<!DOCTYPE html>
<html lang="en-us">

  <head>
  <!-- Google Analytics -->
  <script async src="https://www.googletagmanager.com/gtag/js?id=UA-164212227-2"></script>
  <script>
  window.dataLayer = window.dataLayer || [];
  function gtag(){dataLayer.push(arguments);}
  gtag('js', new Date());
  gtag('config', 'UA-164212227-2');
  </script>
  <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
  <!--[if IE]><meta http-equiv="X-UA-Compatible" content="IE=edge, chrome=1" /><![endif]-->
  <meta name="viewport" content="width=device-width, initial-scale=1" />
  <meta name="description" content="Doyensec's Blog :: Doyensec is an independent security research and development company focused on vulnerability discovery and remediation." />
  <meta name="keywords" content="doyensec, doyensecurity, security, infosec, vulnerability, exploit, offensive, pentest, pentester, hacking, auditing, reverse engineering, automation" />
  <meta name="robots" content="index, follow" />
  <meta name="robots" content="noodp" />
  <meta name="robots" content="noydir" />

  <title>
    
      On Bypassing eBPF Security Monitoring &middot; Doyensec's Blog
    
  </title>

  <!-- CSS -->
  <link rel="stylesheet" href="/public/css/poole.css">
  <link rel="stylesheet" href="/public/css/syntax.css">
  <link rel="stylesheet" href="/public/css/hyde.css">
  <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=PT+Sans:400,400italic,700|Abril+Fatface">

  <!-- Icons -->
  <link rel="apple-touch-icon-precomposed" sizes="144x144" href="/public/images/apple-touch-icon-144-precomposed.png">
  <link rel="shortcut icon" href="/public/images/favicon.ico">

  <!-- RSS -->
  <link rel="alternate" type="application/rss+xml" title="RSS" href="/atom.xml">
</head>


  <body class="theme-base-doyensec">

    <div class="sidebar">
  <div class="navbar-brand"><a href="/index.html"><img src="/public/images/logo.png" class="navbar-brand__image"></a></div>
  <div class="container sidebar-sticky">
    <div class="sidebar-about">
      <p class="lead">We are <a href="https://doyensec.com" target="_blank">security engineers</a> who break bits and tell stories.<br><br>Visit us<br><a href="https://doyensec.com" target="_blank">doyensec.com</a><br><br>Follow us<br><a href="https://twitter.com/doyensec" target="_blank">@doyensec</a><br><br>Engage us<br><a href="mailto:info@doyensec.com" target="_blank">info@doyensec.com</a><br><br></p>
    </div>

    <nav class="sidebar-nav">
      

      
      
        
          
        
      
        
      
        
          
        
      
        
          
        
      
        
          
        
      
        
          
        
      
        
          
        
      
        
          
        
      
        
          
        
      
        
          
        
      
        
          
        
      
        
          
        
      
        
          
        
      
        
          
        
      
    </nav>
    <h4 id="archive-header">Blog Archive</h4>
    <ul class="posts-archive">
            
            
            
              <li>
                <a class="yearBtn" data-year="2017" href="#">2017</a>
                <div class="postsMonths" id="posts-in-2017" style="display:none">
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                      <span class="post-entry" data-date="1511737200" data-url="/2017/11/27/we-are-hiring.html" data-title="We're hiring - Join Doyensec!"></span>
                    
                  
                    
                    
                    
                      <span class="post-entry" data-date="1510700400" data-url="/2017/11/15/osx-spotlight.html" data-title="Staring into the Spotlight"></span>
                    
                  
                    
                    
                    
                      <span class="post-entry" data-date="1501711200" data-url="/2017/08/03/electron-framework-security.html" data-title="Modern Alchemy: Turning XSS into RCE"></span>
                    
                  
                    
                    
                    
                      <span class="post-entry" data-date="1488409200" data-url="/2017/03/02/training-burp.html" data-title="Developing Burp Suite Extensions training"></span>
                    
                  
                </div>
              </li>
            
              <li>
                <a class="yearBtn" data-year="2018" href="#">2018</a>
                <div class="postsMonths" id="posts-in-2018" style="display:none">
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                      <span class="post-entry" data-date="1541372400" data-url="/2018/11/05/burp-rest-api-v2.html" data-title="Introducing burp-rest-api v2"></span>
                    
                  
                    
                    
                    
                      <span class="post-entry" data-date="1531951200" data-url="/2018/07/19/instrumenting-electron-app.html" data-title="Instrumenting Electron Apps for Security Testing"></span>
                    
                  
                    
                    
                    
                      <span class="post-entry" data-date="1527112800" data-url="/2018/05/24/electron-win-protocol-handler-bug-bypass.html" data-title="Electron Windows Protocol Handler MITM/RCE (bypass for CVE-2018-1000006 fix)"></span>
                    
                  
                    
                    
                    
                      <span class="post-entry" data-date="1526508000" data-url="/2018/05/17/graphql-security-overview.html" data-title="GraphQL - Security Overview and Testing Tips"></span>
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                </div>
              </li>
            
              <li>
                <a class="yearBtn" data-year="2019" href="#">2019</a>
                <div class="postsMonths" id="posts-in-2019" style="display:none">
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                      <span class="post-entry" data-date="1572908400" data-url="/2019/11/05/internship-at-doyensec.html" data-title="Internship at Doyensec"></span>
                    
                  
                    
                    
                    
                      <span class="post-entry" data-date="1566424800" data-url="/2019/08/22/modern-password-managers-flag-secure.html" data-title="One Bug To Rule Them All: Modern Android Password Managers and FLAG_SECURE Misuse"></span>
                    
                  
                    
                    
                    
                      <span class="post-entry" data-date="1564610400" data-url="/2019/08/01/common-crypto-bugs.html" data-title="Lessons in auditing cryptocurrency wallets, systems, and infrastructures"></span>
                    
                  
                    
                    
                    
                      <span class="post-entry" data-date="1563746400" data-url="/2019/07/22/jackson-gadgets.html" data-title="Jackson gadgets - Anatomy of a vulnerability"></span>
                    
                  
                    
                    
                    
                      <span class="post-entry" data-date="1562104800" data-url="/2019/07/03/electron-security-workshop.html" data-title="Electron Security Workshop"></span>
                    
                  
                    
                    
                    
                      <span class="post-entry" data-date="1560204000" data-url="/2019/06/11/electronegativity-1.3.html" data-title="Electronegativity 1.3.0 released!"></span>
                    
                  
                    
                    
                    
                      <span class="post-entry" data-date="1556056800" data-url="/2019/04/24/rubyzip-bug.html" data-title="On insecure zip handling, Rubyzip and Metasploit RCE (CVE-2019-5624)"></span>
                    
                  
                    
                    
                    
                      <span class="post-entry" data-date="1554242400" data-url="/2019/04/03/subverting-electron-apps-via-insecure-preload.html" data-title="Subverting Electron Apps via Insecure Preload"></span>
                    
                  
                    
                    
                    
                      <span class="post-entry" data-date="1548284400" data-url="/2019/01/24/electronegativity.html" data-title="Electronegativity is finally out!"></span>
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                </div>
              </li>
            
              <li>
                <a class="yearBtn" data-year="2020" href="#">2020</a>
                <div class="postsMonths" id="posts-in-2020" style="display:none">
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                      <span class="post-entry" data-date="1608159600" data-url="/2020/12/17/psychology-of-remote-work.html" data-title="Psychology of Remote Work"></span>
                    
                  
                    
                    
                    
                      <span class="post-entry" data-date="1607554800" data-url="/2020/12/10/novel-abuses-wifi-direct-mobile-file-transfers.html" data-title="Novel Abuses On Wi-Fi Direct Mobile File Transfers"></span>
                    
                  
                    
                    
                    
                      <span class="post-entry" data-date="1605740400" data-url="/2020/11/19/inql-scanner-v3.html" data-title="InQL Scanner v3 - Just Released!"></span>
                    
                  
                    
                    
                    
                      <span class="post-entry" data-date="1599602400" data-url="/2020/09/09/fuzzilli-jerryscript.html" data-title="Fuzzing JavaScript Engines with Fuzzilli"></span>
                    
                  
                    
                    
                    
                      <span class="post-entry" data-date="1597874400" data-url="/2020/08/20/playframework-csrf-bypass.html" data-title="CSRF Protection Bypass in Play Framework"></span>
                    
                  
                    
                    
                    
                      <span class="post-entry" data-date="1591826400" data-url="/2020/06/11/inql-scanner-v2.html" data-title="InQL Scanner v2 is out!"></span>
                    
                  
                    
                    
                    
                      <span class="post-entry" data-date="1589407200" data-url="/2020/05/14/asn1fuzz.html" data-title="Fuzzing TLS certificates from their ASN.1 grammar"></span>
                    
                  
                    
                    
                    
                      <span class="post-entry" data-date="1588197600" data-url="/2020/04/30/polymorphic-images-for-xss.html" data-title="Researching Polymorphic Images for XSS on Google Scholar"></span>
                    
                  
                    
                    
                    
                      <span class="post-entry" data-date="1586296800" data-url="/2020/04/08/libressl-fuzzer.html" data-title="LibreSSL and OSS-Fuzz"></span>
                    
                  
                    
                    
                    
                      <span class="post-entry" data-date="1585177200" data-url="/2020/03/26/graphql-scanner.html" data-title="InQL Scanner"></span>
                    
                  
                    
                    
                    
                      <span class="post-entry" data-date="1584313200" data-url="/2020/03/16/vscode_codeexec.html" data-title="Don't Clone That Repo: Visual Studio Code^2 Execution"></span>
                    
                  
                    
                    
                    
                      <span class="post-entry" data-date="1583103600" data-url="/2020/03/02/gravitational-audit.html" data-title="2019 Gravitational Security Audit Results"></span>
                    
                  
                    
                    
                    
                      <span class="post-entry" data-date="1582498800" data-url="/2020/02/24/electron-updater-update-signature-bypass.html" data-title="Signature Validation Bypass Leading to RCE In Electron-Updater"></span>
                    
                  
                    
                    
                    
                      <span class="post-entry" data-date="1582066800" data-url="/2020/02/19/solokeys-audit.html" data-title="Security Analysis of the Solo Firmware"></span>
                    
                  
                    
                    
                    
                      <span class="post-entry" data-date="1580684400" data-url="/2020/02/03/heap-exploit.html" data-title="Heap Overflow in F-Secure Internet Gatekeeper"></span>
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                </div>
              </li>
            
              <li>
                <a class="yearBtn" data-year="2021" href="#">2021</a>
                <div class="postsMonths" id="posts-in-2021" style="display:none">
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                      <span class="post-entry" data-date="1621461600" data-url="/2021/05/20/graphql-csrf.html" data-title="That single GraphQL issue that you keep missing"></span>
                    
                  
                    
                    
                    
                      <span class="post-entry" data-date="1615417200" data-url="/2021/03/11/regexploit.html" data-title="Regexploit: DoS-able Regular Expressions"></span>
                    
                  
                    
                    
                    
                      <span class="post-entry" data-date="1613430000" data-url="/2021/02/16/electron-apis-misuse.html" data-title="Electron APIs Misuse: An Attacker’s First Choice"></span>
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                </div>
              </li>
            
              <li>
                <a class="yearBtn" data-year="2022" href="#">2022</a>
                <div class="postsMonths" id="posts-in-2022" style="display:none">
                  
                    
                    
                    
                      <span class="post-entry" data-date="1668466800" data-url="/2022/11/15/learning-ajp.html" data-title="Let's speak AJP"></span>
                    
                  
                    
                    
                    
                      <span class="post-entry" data-date="1667948400" data-url="/2022/11/09/recruiting-security-researchers.html" data-title="Recruiting Security Researchers Remotely"></span>
                    
                  
                    
                    
                    
                      <span class="post-entry" data-date="1666821600" data-url="/2022/10/27/jupytervscode.html" data-title="Visual Studio Code Jupyter Notebook RCE"></span>
                    
                  
                    
                    
                    
                      <span class="post-entry" data-date="1666044000" data-url="/2022/10/18/cloudsectidbit-dataimport.html" data-title="The Danger of Falling to System Role in AWS SDK Client"></span>
                    
                  
                    
                    
                    
                      <span class="post-entry" data-date="1665439200" data-url="/2022/10/11/ebpf-bypass-security-monitoring.html" data-title="On Bypassing eBPF Security Monitoring"></span>
                    
                  
                    
                    
                    
                      <span class="post-entry" data-date="1665007200" data-url="/2022/10/06/semgrep-codeql.html" data-title="Comparing Semgrep and CodeQL"></span>
                    
                  
                    
                    
                    
                      <span class="post-entry" data-date="1664229600" data-url="/2022/09/27/electron-api-default-permissions.html" data-title="Diving Into Electron Web API Permissions"></span>
                    
                  
                    
                    
                    
                      <span class="post-entry" data-date="1662415200" data-url="/2022/09/06/electrong-launch.html" data-title="ElectroNG, our premium SAST tool released!"></span>
                    
                  
                    
                    
                    
                      <span class="post-entry" data-date="1661292000" data-url="/2022/08/24/intern-experience.html" data-title="My Internship Experience at Doyensec"></span>
                    
                  
                    
                    
                    
                      <span class="post-entry" data-date="1658354400" data-url="/2022/07/21/dependency-confusion.html" data-title="Dependency Confusion"></span>
                    
                  
                    
                    
                    
                      <span class="post-entry" data-date="1654725600" data-url="/2022/06/09/apache-pinot-sqli-rce.html" data-title="Apache Pinot SQLi and RCE Cheat Sheet"></span>
                    
                  
                    
                    
                    
                      <span class="post-entry" data-date="1650924000" data-url="/2022/04/26/vbox-fuzzing.html" data-title="Introduction to VirtualBox security research"></span>
                    
                  
                    
                    
                    
                      <span class="post-entry" data-date="1644966000" data-url="/2022/02/16/h1jack-the-game.html" data-title="H1.Jack, The Game"></span>
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                    
                    
                    
                  
                </div>
              </li>
            
    </ul>
    <p>&copy; 2022 <a href="https://doyensec.com" target="_blank">Doyensec LLC.</a><br><small>[<a title="RSS" href="/atom.xml">RSS Feed</a>]</small></p>
  </div>
</div>


    <div class="content container">
      <div class="post on-bypassing-ebpf-security-monitoring">
  <h1 class="post-title">On Bypassing eBPF Security Monitoring</h1>
  <span class="post-date">11 Oct 2022 - Posted by Lorenzo Stella</span>
  <p>There are many security solutions available today that rely on the <a href="https://ebpf.io/">Extended Berkeley Packet Filter (eBPF)</a> features of the Linux kernel to monitor kernel functions. Such a paradigm shift in the latest monitoring technologies is being driven by a variety of reasons. Some of them are motivated by performance needs in an increasingly cloud-dominated world, among <a href="https://www.youtube.com/watch?v=44nV6Mj11uw">others</a>. The Linux kernel always had kernel tracing capabilities such as <a href="https://docs.kernel.org/trace/kprobes.html">kprobes</a> (2.6.9), <a href="https://www.kernel.org/doc/Documentation/trace/ftrace.txt">ftrace</a> (2.6.27 and later), <a href="https://perf.wiki.kernel.org/index.php/Main_Page">perf</a> (2.6.31), or <a href="https://docs.kernel.org/trace/uprobetracer.html">uprobes</a> (3.5), but with BPF it’s finally possible to run kernel-level programs on events and consequently modify the state of the system, without needing to write a kernel module. This has dramatic implications for any attacker looking to compromise a system and go undetected, opening new areas of research and application. Nowadays, eBFP-based programs are used for <a href="https://blog.cloudflare.com/how-to-drop-10-million-packets/">DDoS mitigations</a>, <a href="https://dl.acm.org/doi/abs/10.1016/j.jnca.2021.103283">intrusion detection</a>, <a href="https://developers.redhat.com/articles/2021/12/16/secure-your-kubernetes-deployments-ebpf">container security</a>, and general observability.</p>

<p>In 2021 <a href="https://goteleport.com/">Teleport</a> introduced a new feature called <a href="https://goteleport.com/blog/enhanced-session-recording/">Enhanced Session Recording</a> to close some monitoring gaps in Teleport’s audit abilities. All issues reported have been promptly fixed, mitigated or documented as described in their <a href="https://goteleport.com/resources/audits/teleport-features-security-audit-q4-2021/">public Q4 2021 report</a>. Below you can see an illustration of how we managed to bypass eBPF-based controls, along with some ideas on how red teams or malicious actors could evade these new intrusion detection mechanisms. These techniques can be generally applied to other targets while attempting to bypass any security monitoring solution based on eBPF:</p>

<ul id="markdown-toc">
  <li><a href="#a-few-words-on-how-ebpf-works" id="markdown-toc-a-few-words-on-how-ebpf-works">A few words on how eBPF works</a></li>
  <li><a href="#common-shortcomings--potential-bypasses-here-be-dragons" id="markdown-toc-common-shortcomings--potential-bypasses-here-be-dragons">Common shortcomings &amp; potential bypasses (here be dragons)</a>    <ul>
      <li><a href="#1-understand-which-events-are-caught" id="markdown-toc-1-understand-which-events-are-caught">1. Understand which events are caught</a>        <ul>
          <li><a href="#11-execution-bypasses" id="markdown-toc-11-execution-bypasses">1.1 Execution bypasses</a></li>
          <li><a href="#12-network-bypasses" id="markdown-toc-12-network-bypasses">1.2 Network bypasses</a></li>
        </ul>
      </li>
      <li><a href="#2-delayed-execution" id="markdown-toc-2-delayed-execution">2. Delayed execution</a></li>
      <li><a href="#3-evade-scoped-event-monitoring-based-on-cgroup" id="markdown-toc-3-evade-scoped-event-monitoring-based-on-cgroup">3. Evade scoped event monitoring based on <code class="language-plaintext highlighter-rouge">cgroup</code></a></li>
      <li><a href="#4-memory-limits-and-loss-of-events" id="markdown-toc-4-memory-limits-and-loss-of-events">4. Memory limits and loss of events</a></li>
      <li><a href="#5-never-trust-the-userspace" id="markdown-toc-5-never-trust-the-userspace">5. Never trust the userspace</a></li>
      <li><a href="#6-abuse-the-lack-of-seccomp-bpf--kernel-discrepancies" id="markdown-toc-6-abuse-the-lack-of-seccomp-bpf--kernel-discrepancies">6. Abuse the lack of <code class="language-plaintext highlighter-rouge">seccomp-bpf</code> &amp; kernel discrepancies</a></li>
      <li><a href="#7-interfere-with-the-agents" id="markdown-toc-7-interfere-with-the-agents">7. Interfere with the agents</a></li>
    </ul>
  </li>
</ul>

<h2 id="a-few-words-on-how-ebpf-works">A few words on how eBPF works</h2>
<p><br />
<img src="../../../public/images/eBPF.png" alt="eBPF schema" align="center" style="display: block; margin-left: auto; margin-right: auto;" /></p>

<p>Extended BPF programs are written in a high-level language and compiled into eBPF bytecode using a toolchain. A user mode application loads the bytecode into the kernel using the <code class="language-plaintext highlighter-rouge">bpf()</code> syscall, where the eBPF verifier will perform a number of checks to ensure the program is “safe” to run in the kernel. This verification step is critical — eBPF exposes a path for unprivileged users to execute in ring 0. Since allowing unprivileged users to run code in the kernel is a ripe attack surface, several pieces of research in the past focused on local privilege exploitations (LPE), which we won’t cover in this blog post.
After the program is loaded, the user mode application attaches the program to a hook point that will trigger the execution when a certain hook point (event) is hit (occurs). The program can also be JIT compiled into native assembly instructions in some cases. User mode applications can interact with, and get data from, the eBPF program running in the kernel using eBPF maps and eBPF helper functions.</p>

<h2 id="common-shortcomings--potential-bypasses-here-be-dragons">Common shortcomings &amp; potential bypasses (here be dragons)</h2>

<h3 id="1-understand-which-events-are-caught">1. Understand which events are caught</h3>
<p>While eBPF is fast (much faster than <a href="https://linux.die.net/man/8/auditd">auditd</a>), there are plenty of interesting areas that can’t be reasonably instrumented with BPF due to performance reasons. Depending on what the security monitoring solution wants to protect the most (e.g., network communication vs executions vs filesystem operations), there could be areas where excessive probing could lead to a performance overhead pushing the development team to ignore them. This depends on how the endpoint agent is designed and implemented, so carefully auditing the code security of the eBPF program is paramount.</p>

<h4 id="11-execution-bypasses">1.1 Execution bypasses</h4>
<p>By way of example, a simple monitoring solution could decide to hook only the <code class="language-plaintext highlighter-rouge">execve</code> system call. Contrary to popular belief, multiple ELF-based Unix-like kernels don’t need a file on disk to load and run code, even if they usually require one. One way to achieve this is by using a technique called reflective loading. Reflective loading is an important post-exploitation technique usually used to avoid detection and execute more complex tools in locked-down environments. The man page for <code class="language-plaintext highlighter-rouge">execve()</code> states: “<em><code class="language-plaintext highlighter-rouge">execve()</code> executes the program pointed to by filename…</em>”, and goes on to say that “<em>the text, data, bss, and stack of the calling process are overwritten by that of the program loaded</em>”. This overwriting doesn’t necessarily constitute something that the Linux kernel must have a monopoly over, unlike filesystem access, or any number of other things. Because of this, the <code class="language-plaintext highlighter-rouge">execve()</code> system call can be mimicked in userland with a minimal difficulty. Creating a new process image is therefore a simple matter of:</p>
<ul>
  <li>cleaning out the address space;</li>
  <li>checking for, and loading, the dynamic linker;</li>
  <li>loading the binary;</li>
  <li>initializing the stack;</li>
  <li>determining the entry point and</li>
  <li>transferring control of execution.</li>
</ul>

<p>By following these six steps, a new process image can be created and run. Since this technique was <a href="https://grugq.github.io/docs/ul_exec.txt">initially reported in 2004</a>, the process has nowadays been pioneered and streamlined by OTS post-exploitation tools. As anticipated, an eBPF program hooking <code class="language-plaintext highlighter-rouge">execve</code> would not be able to catch this, since this custom userland <code class="language-plaintext highlighter-rouge">exec</code> would effectively replace the existing process image within the current address space with a new one. In this, userland exec mimics the behavior of the system call <code class="language-plaintext highlighter-rouge">execve()</code>. However, because it operates in userland, the kernel process structures which describe the process image remain unchanged.</p>

<p>Other system calls may go unmonitored and decrease the detection capabilities of the monitoring solution. Some of these are <code class="language-plaintext highlighter-rouge">clone</code>, <code class="language-plaintext highlighter-rouge">fork</code>, <code class="language-plaintext highlighter-rouge">vfork</code>, <code class="language-plaintext highlighter-rouge">creat</code>, or <code class="language-plaintext highlighter-rouge">execveat</code>.</p>

<p>Another potential bypass may be present if the BPF program is naive and trusts the <code class="language-plaintext highlighter-rouge">execve</code> syscall argument referencing the complete path of the file that is being executed. An attacker could create symbolic links of Unix binaries in different locations and execute them - thus tampering with the logs.</p>

<h4 id="12-network-bypasses">1.2 Network bypasses</h4>

<p>Not hooking all the network-related syscalls can have its own set of problems. Some monitoring solutions may only want to hook the EGRESS traffic, while an attacker could still send data to a non-allowed host abusing other network-sensitive operations (<a href="https://code.woboq.org/linux/linux/security/apparmor/include/audit.h.html#78">see</a> <code class="language-plaintext highlighter-rouge">aa_ops</code> at <code class="language-plaintext highlighter-rouge">linux/security/apparmor/include/audit.h:78</code>) related to INGRESS traffic:</p>
<ul>
  <li><code class="language-plaintext highlighter-rouge">OP_BIND</code>, the <code class="language-plaintext highlighter-rouge">bind()</code> function shall assign a local socket address to a socket identified by descriptor socket that has no local socket address assigned.</li>
  <li><code class="language-plaintext highlighter-rouge">OP_LISTEN</code>, the <code class="language-plaintext highlighter-rouge">listen()</code> function shall mark a connection-mode socket, specified by the socket argument, as accepting connections.</li>
  <li><code class="language-plaintext highlighter-rouge">OP_ACCEPT</code>, the <code class="language-plaintext highlighter-rouge">accept()</code> function shall extract the first connection on the queue of pending connections, create a new socket with the same socket type protocol and address family as the specified socket, and allocate a new file descriptor for that socket.</li>
  <li><code class="language-plaintext highlighter-rouge">OP_RECVMSG</code>, the <code class="language-plaintext highlighter-rouge">recvmsg()</code> function shall receive a message from a connection-mode or connectionless-mode socket.</li>
  <li><code class="language-plaintext highlighter-rouge">OP_SETSOCKOPT</code>, the <code class="language-plaintext highlighter-rouge">setsockopt()</code> function shall set the option specified by the option_name argument, at the protocol level specified by the level argument, to the value pointed to by the option_value argument for the socket associated with the file descriptor specified by the socket argument. Interesting options for attackers are <code class="language-plaintext highlighter-rouge">SO_BROADCAST</code>, <code class="language-plaintext highlighter-rouge">SO_REUSEADDR</code>, <code class="language-plaintext highlighter-rouge">SO_DONTROUTE</code>.</li>
</ul>

<p>Generally, the network monitoring should look at all socket-based operations similarly to <em>AppArmor</em>.</p>

<p>In case the same local user has mixed monitored and unmonitored console sessions, it could be possible for an attacker in a monitored session to leverage open file descriptors and sockets to send data to restricted hosts. In 2020 some versions of Linux kernels had introduced a <a href="https://man7.org/linux/man-pages/man2/pidfd_getfd.2.html">new system call</a> to achieve this called <code class="language-plaintext highlighter-rouge">pidfd_getfd</code>. A small number of operating systems (like Ubuntu) implement the <a href="https://www.kernel.org/doc/Documentation/security/Yama.txt">Yama</a> kernel module that limit file descriptor access to only child-parent processes. A PoC code for using this function is available on Github (<a href="https://github.com/TheZ3ro/fdstealer">TheZ3ro/fdstealer</a>).</p>

<p><img src="../../../public/images/fd-stealing.png" alt="File descriptor stealing using pidfd_getfd" align="center" style="display: block; margin-left: auto; margin-right: auto;" /></p>

<h3 id="2-delayed-execution">2. Delayed execution</h3>
<p>If only active console sessions are monitored, eBPF programs may only live for the time span of the session. By delaying the execution of a command (through <code class="language-plaintext highlighter-rouge">sleep</code>, <code class="language-plaintext highlighter-rouge">timeout</code>, <code class="language-plaintext highlighter-rouge">sar</code>, <code class="language-plaintext highlighter-rouge">vmstat</code>, <code class="language-plaintext highlighter-rouge">inotifywait</code>, <code class="language-plaintext highlighter-rouge">at</code>, <code class="language-plaintext highlighter-rouge">cron</code> …) and quitting the session, it could be possible to completely evade the solution.</p>

<h3 id="3-evade-scoped-event-monitoring-based-on-cgroup">3. Evade scoped event monitoring based on <code class="language-plaintext highlighter-rouge">cgroup</code></h3>
<p>A security monitoring solution may only be interested in auditing a specific user or cgroup (such in the context of a remote console session). Taking Teleport as an example, it achieves this by correlating every event to a session with control groups (<code class="language-plaintext highlighter-rouge">cgroupv2</code> in particular). Control grouping is a Linux kernel feature to limit access to resources to a group of processes. It is used in many containerization technologies (behind the scenes Docker creates a set of namespaces and control groups for the container) and its peculiarity is that all child processes will keep the id of the parent process. When Teleport starts an SSH session, it first re-launches itself and places itself within a cgroup. This allows not only that process, but all future processes that Teleport launches, to be tracked with a unique ID. The BPF programs that Teleport runs have been updated to also emit the cgroup ID of the program executing them. The BPF script checks the value returned by <code class="language-plaintext highlighter-rouge">bpf_get_current_cgroup_id()</code> and only cares about the important session cgroup. The simplest evasion to this auditing strategy would be changing your cgroup ID, but an attacker needs to be root to achieve this. Meddling with the cgroupv2 pseudo file system or abusing PAM configuration are also potential opportunities to affect the cgroup/session correlation.</p>

<p>Another technique involves being reallocated by init. In the case of Teleport, when the <code class="language-plaintext highlighter-rouge">bash</code> process spawned by the session dies, its child processes become orphans and the Teleport process terminates its execution. When a child process becomes an orphan, it can be assigned to a different cgroup by the operating system under certain conditions (not having a tty, being a process group leader, joining a new process session). This allows an attacker to bypass the restrictions in place. The following PoC is an example of a bypass for this design:</p>

<ol>
  <li>Open a new eBPF-monitored session</li>
  <li>Start <a href="https://github.com/tmux/tmux/wiki">tmux</a> by executing the <code class="language-plaintext highlighter-rouge">tmux</code> command</li>
  <li>Detach from <code class="language-plaintext highlighter-rouge">tmux</code> by pressing <code class="language-plaintext highlighter-rouge">CTRL+B</code> and then <code class="language-plaintext highlighter-rouge">D</code></li>
  <li>Kill the bash process that is <code class="language-plaintext highlighter-rouge">tmux</code>’s parent</li>
  <li>Re-attach to the <code class="language-plaintext highlighter-rouge">tmux</code> process by executing <code class="language-plaintext highlighter-rouge">tmux attach</code>. The process tree will now look like this:</li>
</ol>

<p><img src="../../../public/images/cgroup-evasion.png" alt="CGroupV2 evasion PoC" align="center" style="display: block; margin-left: auto; margin-right: auto;" /></p>

<p>As another attack avenue, leveraging processes run by different local users/<code class="language-plaintext highlighter-rouge">cgroupv2</code> on the machine (abusing other daemons, delegating systemd) can also help an attacker evade this. This aspect obviously depends on the system hosting the monitoring solution. Protecting against this is tricky, since even if <code class="language-plaintext highlighter-rouge">PR_SET_CHILD_SUBREAPER</code> is set to ensure that the descendants can’t re-parent themselves to init, if the ancestor reaper dies or is killed (DoS), then processes in that service can escape their cgroup “container”. Any compromise of this privileged service process (or malfeasance by it) allows it to kill its hierarchy manager process and escape all control.</p>

<h3 id="4-memory-limits-and-loss-of-events">4. Memory limits and loss of events</h3>
<p>BPF programs have a lot of constraints. Only 512 bytes of stack space are reserved for the eBPF program. Variables will get hoisted and instantiated at the start of execution, and if the script tries to dump syscall arguments or <code class="language-plaintext highlighter-rouge">pt-regs</code>, it will run out of stack space very quickly. If no workaround on the instruction limit is set, it could be possible to push the script into retrieving something too big to ever fit on the stack, losing visibility very soon when the execution gets complicated. But even when workarounds are used (e.g., when using multiple probes to trace the same events but capture different data, or split your code into multiple programs that call each other using a program map) there still may be a chance to abuse it. BPF programs are not meant to be run forever, but they have to stop at some point. By way of example, if a monitoring solution is running on CentOS 7 and trying to capture a process arguments and its environment variables, the emitted event could have too many argv and too many envp. Even in that case, you may miss some of them because the loop stops earlier. In these cases, the event data will be truncated. It’s important to note that these limitations are different based on the kernel where BPF is being run, and how the endpoint agent is written.</p>

<p>Another peculiarity of eBPFs is that they’ll drop events if they can not be consumed fast enough, instead of dragging down the performance of the entire system with it. An attacker could abuse this by generating a sufficient number of events to fill up the perf ringbuffer and overwrite data before the agent can read it.</p>

<h3 id="5-never-trust-the-userspace">5. Never trust the userspace</h3>
<p>The kernel-space understanding of a <code class="language-plaintext highlighter-rouge">pid</code> is not the same as the user-space understanding of a <code class="language-plaintext highlighter-rouge">pid</code>. If the eBPF script is trying to identify a file, the right way would be to get the inode number and device number, while a file descriptor won’t be as useful. Even in that case, probes could be subject to TOCTOU issues since they’ll be sending data to user mode that can easily change. If the script is instead tracing syscalls directly (using <code class="language-plaintext highlighter-rouge">tracepoint</code> or <code class="language-plaintext highlighter-rouge">kprobe</code>) it is probably stuck with file descriptors and it could be possible to obfuscate executions by playing around with the current working directory and file descriptors, (e.g., by combining <code class="language-plaintext highlighter-rouge">fchdir</code>, <code class="language-plaintext highlighter-rouge">openat</code>, and <code class="language-plaintext highlighter-rouge">execveat</code>).</p>

<h3 id="6-abuse-the-lack-of-seccomp-bpf--kernel-discrepancies">6. Abuse the lack of <code class="language-plaintext highlighter-rouge">seccomp-bpf</code> &amp; kernel discrepancies</h3>
<p>eBPF-based monitoring solutions should protect themselves by using <a href="https://www.kernel.org/doc/html/v4.19/userspace-api/seccomp_filter.html">seccomp-BPF</a> to permanently drop the ability to make the <code class="language-plaintext highlighter-rouge">bpf()</code> syscall before spawning a console session. If not, an attacker will have the ability to make the bpf() syscall to unload the eBPF programs used to track execution. Seccomp-BPF uses BPF programs to filter arbitrary system calls and their arguments (constants only, no pointer dereference).</p>

<p>Another thing to keep in mind when working with kernels, is that interfaces aren’t guaranteed to be consistent and stable. An attacker may abuse eBPF programs if they are not run on verified kernel versions. Usually, conditional compilation for a different architecture is very convoluted for these programs and you may find that the variant for your specific kernel is not targeted correctly. One common pitfall of using seccomp-BPF is filtering on system call numbers without checking the <code class="language-plaintext highlighter-rouge">seccomp_data-&gt;arch</code> <a href="https://www.kernel.org/doc/Documentation/prctl/seccomp_filter.txt">BPF program argument</a>. This is because on any architecture that supports multiple system call invocation conventions, the system call numbers may vary based on the specific invocation. If the numbers in the different calling conventions overlap, then checks in the filters may be abused. It is therefore important to ensure that the differences in <code class="language-plaintext highlighter-rouge">bpf()</code> invocations for each newly supported architecture are taken into account by the seccomp-BPF filter rules.</p>

<h3 id="7-interfere-with-the-agents">7. Interfere with the agents</h3>
<p>Similarly to (6), it may be possible to interfere with the eBPF program loading in different ways, such as targeting the eBPF compiler libraries (<a href="https://github.com/iovisor/bcc">BCC</a>’s <code class="language-plaintext highlighter-rouge">libbcc.so</code>) or adapting other shared libraries preloading methods to tamper with the behavior of legit binaries of the solution, ultimately performing harmful actions. In case an attacker succeeds in altering the solution’s host environment, they can add in front of the <code class="language-plaintext highlighter-rouge">LD_LIBRARY_PATH</code>, a directory where they saved a malicious library having the same <code class="language-plaintext highlighter-rouge">libbcc.so</code> name and exporting all the symbols used (to avoid a runtime linkage error). When the solution starts, instead of the legit bcc library, it gets linked with the malicious library. Defenses against this may include using statically linked programs, linking the library with the full path, or running the program into a controlled environment.</p>

<p>Many thanks to the whole <a href="https://goteleport.com/">Teleport Security Team</a>, <a href="https://tmpout.sh/2/4.html">@FridayOrtiz</a>, <a href="https://twitter.com/th3zer0">@Th3Zer0</a>, &amp; <a href="https://mobile.twitter.com/alessandrogario">@alessandrogario</a> for the inspiration and feedback while writing this blog post.</p>

</div>
<div class="related">
  <h3>Other relevant posts:</h3> 
  <ul class="related-posts">
    
    
    
                           
    
    
    
                           
    
    
    
                           
    
    
    
                           
    
    
    
    
       
                           
    
    
    
                           
    
    
    
                           
    
    
    
                           
    
    
    
                           
    
    
    
                           
    
    
    
                           
    
    
    
                           
    
    
    
                           
    
    
    
                           
    
    
    
                           
    
    
    
                           
    
    
    
                           
    
    
    
                           
    
    
    
                           
    
    
    
                           
    
    
    
                           
    
    
    
                           
    
    
    
                           
    
    
    
                           
    
    
    
                           
    
    
    
                           
    
    
    
                           
    
    
    
    
      
      <li>
        <h3>
          <a href="/2020/03/02/gravitational-audit.html">
            2019 Gravitational Security Audit Results
            <small>02 Mar 2020</small>
          </a>
        </h3>
      </li>
       
                           
    
    
    
                           
    
    
    
    
      
      <li>
        <h3>
          <a href="/2020/02/19/solokeys-audit.html">
            Security Analysis of the Solo Firmware
            <small>19 Feb 2020</small>
          </a>
        </h3>
      </li>
       
                           
    
    
    
                           
    
    
    
                           
    
    
    
                           
    
    
    
                           
    
    
    
                           
    
    
    
                           
    
    
    
                           
    
    
    
                           
    
    
    
                           
    
    
    
                           
    
    
    
                           
    
    
    
                           
    
    
    
                           
    
    
    
                           
    
    
    
                           
    
    
    
                           
    
    
    
                           
    
    
    
                           
    
  </ul>
</div>

    </div>

  <script src="/public/js/jquery.min.js"></script>
  <script src="/public/js/main.js"></script>
  </body>
</html>
